"""

"""

#!/usr/bin/python
# -*- coding: utf-8 -*-
import time
import os
import sys

print("IN GRIDEPISTASISIPY")
print("1) Import of config,migsession,logger and jobfragmentation")
sys.path.append('Configuration/')
import epistasisconfiguration as configuration
#epistasis.py is defined as the main_script in Config and included in create_init_job

sys.path.append('Gridinterface/')
import migsession
from mylogger import log

#sys.path.append('misc/')
sys.path.append('Epicalc/')
import epicalcP
from jobfragmentation import fragment_epistasis, get_job_specs

#print("sysarg")
#print(sys.argv[1])

class GridEpistasis:
    """Start an epistasis procedure either executed on grid og locally controlled in mig_session."""
    print("2) NEW_EPISTASIS in  '__main__' creates a new instance of class GridEpistasis")
    def __init__(self, debug_mode=False):
        self.jobs_done = []
        self.epistasis_jobs = []
        self.all_jobs = []
        self.epistasis_status = 'idle'
        self.num_jobs = 0
        self.logfile = ""
        self.summary_joint = ""
        self.debug_mode = debug_mode
        self.main_output_dir = ""
        #self.mig_session = ""

# ########## UPDATE / STATUS ###############
    
    def get_epistasis_status(self):
        """Return the status of the jobs and a progress indicator to be displayed in GUI."""
	"""Called where???"""
	print("get_epistasis_status(self)")
        self.mig_session.update_jobs(self.epistasis_jobs)
        for j in self.epistasis_jobs:
            if j['status']['STATUS'] == 'FINISHED':
                self.jobs_done.append(j)
                self.epistasis_jobs.remove(j)
                #self.mig_session.handle_output(j)
                output_files = self.mig_session.handle_output(j)
                for f in output_files:
                    self.extract_output(f,  self.main_output_dir+j['results_dir'])

        if self.num_jobs == len(self.jobs_done):
            log(self.logfile, 'All jobs completed', self.debug_mode)
            self.epistasis_status = 'finished'

        progress_str = str(len(self.jobs_done)) + '/'\
             + str(self.num_jobs)
        status_lines = self.create_status_feed(self.epistasis_jobs)
        status_lines.extend(self.create_status_feed(self.jobs_done))
        status = ''
        for line in status_lines:
            status += line + '\n'
        return (status, progress_str)


    def create_status_feed(self, jobs):
        """Return a status string for each job"""
        feed = []
        for j in jobs:
            line = self.create_status_str(j)
            feed.append(line)
	print("create_status_feed(self, jobs)")
	print(feed)
        return feed


    def create_status_str(self, job):
        """Return a status string for a job"""
        status_str = 'Epistasis \t - class '
        for val in job['class']:
            status_str += str(val) + ' ' 
        status_str += '\t - genes  '
        for gene in job["gene_list"]:
            status_str += str(gene) + ' ' 
        status_str += '-'+ '\t' + job['status']['STATUS']
        if job['status']['STATUS'] != "EXECUTING":
            status_str += "\t"
        status_str += ' \t started:'+job["started"]+'\t \t ended:'+job["finished"]
        return status_str
	print("create_status_str(self, job)")

    def monitor_epistasis(self):
        """Monitor the epistasis procedure."""
	print("monitor_epistasis(self)")
        jobs_done = []
        jobs = self.epistasis_jobs


#	print jobs
    # mylogger.logprint(logfile, "Started monitoring")

        while True:
            try:
                self.mig_session.update_jobs(jobs)
                for j in jobs:
                    if j['status']['STATUS'] == 'FINISHED':
                        output_files = self.mig_session.handle_output(j)
                        for f in output_files:
                            self.extract_output(f,  self.main_output_dir+j['results_dir'])
                        
                        jobs_done.append(j)
                        jobs.remove(j)
                        
                        #log(self.logfile, 'Job ' + j['id'] + ' done.', self.debug_mode)
                        print 'Job ' + j['id'] + ' done.'
                if jobs == []:

                # mylogger.logprint(logfile, "All jobs completed")
                    self.print_status(jobs_done)
                    print 'all jobs completed'
                    return
                self.print_status(jobs)
                self.print_status(jobs_done)
                time.sleep(configuration.monitor_polling_frequency)
            except KeyboardInterrupt:
                print 'User initiated cancellation of jobs'
                self.mig_session.cancel_jobs(jobs)
                return
        return jobs_done


    def extract_output(self,tar_file_path, dest_dir):
        import tarfile
        
        new_dir = dest_dir+tar_file_path.split("/")[-1].strip(".tar.gz")
        if not os.path.exists(new_dir):
            os.mkdir(new_dir)
        prog_files = tarfile.open(tar_file_path, "r")
            
        prog_files.extractall(path=new_dir)
        prog_files.close()

	print("extract_output(self,tar_file_path, dest_dir)")


# ######### START EPISTASIS ############
	"""First call.Building jobs"""
    def start_epistasis( 
        self,
        selection_variable=configuration.default_selection_variable_index,
	#"""above is the partition used"""
        selection_variable_values = configuration.default_variable_values,
	#"""above is the classes in the partion to be analysed"""
        genelist=configuration.default_gene_list,
        traitlist=configuration.default_trait_list,
        data=configuration.data_file,
#	data2 = configuration.data_file2,
        output_dir=configuration.output_dir,#this is epistasis/
        job_size=configuration.default_job_size,
        local_mode=False,
        ):
        """Start the epistasis procedure."""

	print("5) GRIDE Overall entries (e.g. genelist) are defined by call to 'create_epistasis_jobs'")
	#project_dir == job_result_dir return from call below 				i.e. EpistasisFiles13_3_2011_183948
        epi_jobs, project_dir = create_epistasis_jobs(
            job_size,
            genelist=genelist,
            traitlist=traitlist,
	    #this is the partition used
            selection_var=selection_variable,
	    #this is the classes used
            variable_values=selection_variable_values,
            data_file=data,
#            data_file2=data2,
            output_dir=output_dir,#this is epistasis/
            run_local=local_mode,
            )

        self.main_output_dir = output_dir
	"""ouput_dir = epifiles/"""	
	print("15) Now all jobs are defined")

        if not output_dir[-1] == "/":
            output_dir += "/"
        output_dir = output_dir+project_dir
#	print(output_dir)
	"""Now output_dir is epifiles/EpistasisFiles20_11_2010_191238/, that is the directory for all EXTRACTED jobs"""
	"""The logfile.txt is placed in this directory"""
        self.logfile = output_dir+configuration.logfile_name
        
        self.summary_joint = output_dir+configuration.summary_name
        """her skal vi have oprettet summary file for generelle output bla med oplysibnger om manglende classes"""
#        summaryall = open(summary_joint, "a")
        
        
#    logfile.write(entry)

        
	print("16) GRIDE Extraction directory is created")

        os.mkdir(output_dir)

	"""Switching to mig-session"""
	print("17) Switching to mig-session")

        self.mig_session = \
            migsession.MigSession(output_dir, self.logfile, local_mode, self.debug_mode)
	"""self.mig_session is the adress for the migsession.py"""

	"""After the mig-session is set above (local/mig) the next call creates the jobs in mig_jobs shell"""
	print("20) GRIDE Return from MIG. Jobs are created in the MIG")
        self.mig_session.create_mig_jobs(epi_jobs, configuration.Epistasis_working_dir)
	print("jobs has been created")
        self.epistasis_jobs.extend(epi_jobs)
        self.all_jobs.extend(epi_jobs)
        self.num_jobs = len(epi_jobs)


# ######### STOP /CANCEL ##############

    def stop_epistasis(self):
        """Stop the epistasis procedure."""
        self.mig_session.cancel_jobs(self.epistasis_jobs)

# ##### PRINT ###########

    def print_jobs(self, jobs):
        """Print jobs."""
        for i in range(len(jobs)):
            print 'job ' + str(i) + ' : ' + str(jobs[i])

    def print_status(self, jobs):
        """Print job status."""
        full_str = []
        for j in jobs:
            status_str = 'Job : ' + j['id'] + '\t' + j['status'
                    ]['STATUS']
            print status_str
            full_str.append(status_str)
        return full_str

# ### CLEAN UP ########

    def clean_up_epistasis(self):
        """Delete files used in the epistasis procedure that are no longer needed."""
        self.mig_session.clean_up(self.all_jobs)

# ##### CREATE JOBS#############

"""Here job-contents and files are defined, but is not executed (this is done from mig_session)"""
def create_epistasis_jobs(
    job_size,
    genelist,
    traitlist,
    selection_var,
    variable_values,
    data_file,
#    data_file2,
    output_dir,
    run_local=False,
    ):
    
    """This should be moved!!!??!!"""
    print("5a) clean data file")
#    print(output_dir)	
 #   print(os.getcwd())
    partition = configuration.default_selection_variable_index
    traits = configuration.default_trait_list
    genelist = configuration.default_gene_list
    data_file_raw = configuration.data_file.split('/')[-1]

    cleanup_data_raw = epicalcP.get_epi_dataraw(data_file_raw,"Epicalc/",partition,traits,genelist)
    data_trim = cleanup_data_raw[0]
    snp_number = cleanup_data_raw[1]
    partition_use = cleanup_data_raw[2]
    traits_use = cleanup_data_raw[3]

    """Return epistasis jobs that are executed in the epistasis procedure."""
    """ 'jobs' created as 'job' below samples all information needed to run epistasis"""
    """but the programme is not started her, but is propagated in mig-session"""

    print("6) GRIDE Class numbers are called from the imported script 'fragment_epistasis'")
#    print("snp_number")
 #   print(snp_number)
    job_fragments = fragment_epistasis(job_size, values=variable_values)
    """values are simply the class-numbers"""

    """Enumerate the class-values in the selected partition"""
    class_exist_list=data_trim[1:,1].astype(int).tolist()
    class_content = []
    for clch in range(0,len(class_exist_list)):
        clnumber=class_exist_list[0:][clch][0]
        class_content.append(clnumber) 
    classes_exist = set(class_content)
    
    """jobs is a list of contents in initiated jobs (jobs.append(job)"""
    print("7) GRIDE An empty list to contain each job is created called 'jobs'")    
    jobs = []

    print("8) GRIDE ID for the entire session is defined as date and time")    
    time_list = time.localtime(time.time())
    project_tag = str(time_list[2]) + '_' + str(time_list[1]) + '_'\
         + str(time_list[0]) + '_' + str(time_list[3])\
         + str(time_list[4]) + str(time_list[5])

    print("9) GRIDE Individual jobs are defined, the job-number supplied from job_fragments above ")   

    
    for j in job_fragments:#job_fragments delivers classes by exact number

        """This skips non-existing classes, i.e. job is not created. See also procedure in epistasisP.py"""
        class_chosen = set([int(j[0])])
        if class_chosen.issubset(classes_exist) ==False:
#           print("class does not exist")
           #vi skal have en udskrift til summary_joint
           continue


##The job content is defined in 'create_init_job', a dictionary
        job = create_init_job(project_tag)
        classes = j

	print("11b) GRIDE Jobs created in create_init with exact class number" + str(j))
#	print(job)
	#Add the specific class-number to the job 
        job['class'] = classes
        job['data_file_raw'] = configuration.data_file.split('/')[-1]
        job['data_file_trimmed'] = data_trim 
        job['snps_remaining'] = snp_number 
        job['partition_use'] = partition_use 
        job['traits_use'] = traits_use
        
	"""NB the entrance below could not found locally, but may be in use on the grid"""
#        job['user_output_dir'] = output_dir#this is epistasis/MOve this to general job-description?
# is there already, but should then be called below!

	print("12) GRIDE The class-numbered tar.gz-file is defined e.g. epifilesX.tar.gz and added to the job dictionary")
        output_filename = 'epifiles' + str(classes[0]) + '.tar.gz'
	"""and is added to the job-list"""
        job['output_files'] = [output_filename]

	"""Folder in which scripts, epistasis.py, datafile, tar.gz-file and pkl-file evt. is temporary? saved to"""
	"""It is named like this: EpiMigJobFiles1_17_12_2010_1142"""
        job_directory = configuration.tmp_local_job_dir\
             + str(classes[0]) + '_' + project_tag + '/'	
	"""..and kept it in the Epistais_tmp/ directory"""
        job['job_dir'] = configuration.Epistasis_working_dir+job_directory

	"""End of defining tmp-folders""" 

	print("13) GRIDE Folder for extracted files is defined e.g. EpistasisFiles17_12_2010_114021/")
        job_results_dir = configuration.resultsdir_prefix_name + project_tag +"/"
        job['results_dir'] = job_results_dir
	"""This folder contains all the jobs e.g. epifiles1"""
	"""it will be located in the epistasis/ folder eventually"""


#	print("results_dir file er den overordnede fil som alle nummererede epi-resultater skrives til een class ad gangen efter extraction dvs i epifiles:")
#	print("14) Executable for epistasis.py is defined!!! and added to job")
#        job_cmds = ['python '+ job['main_script']]#move to general
	"""job_cmds = python epistasis.py"""
#	print("Job cmds to execute epistasis.py for all jobs created")
#	print(job_cmds)

#        job['commands'] = job_cmds
	"""Flyt til common????"""
	print("14) GRIDE  Defines the location and content of the input_files i.e. scripts and data files") 
        
        input_files = list(configuration.program_files)
	"""The data files is added to the list..."""
#        input_files.append(data_file)
        input_files.append(data_file)
	"""New py-scripts are added, just like R-scripts"""
	"""..and the jobs are updated"""
        job['input_files'] = input_files
#        print(job["input_files"])
####Mig-stuff
        job['resource_specs'] = configuration.resource_specs

#        if run_local:
 #           job['r_bin'] = 'R'
  #      else:
   #         job['r_bin'] = '$R_HOME/bin/R'
#
        jobs.append(job)

#    print(job)
    return jobs, job_results_dir


def create_init_job(project_tag):
    """Define content of a job[..]"""
    """All common stuff should compiled here"""
    "All r-scripts are listed, but not located. This is done the calling function above. Condense???"
    print("11) The dictionary defining all elements to be used in the job including scripts, datafiles and selection of partition, genes and traits")
    init_job = {}#this is a dictionary, not a set, as set cannot be defined this way
#Original in the calling script
    init_job['project_tag'] = project_tag
#    init_job['monomorph_snp'] = monomorf_general
#The definitions below is all call to the configuration script
#    init_job["main_r_file"] = configuration.main_r_file
#    init_job['r_files'] = configuration.r_files
    init_job['p_files'] = configuration.p_files
    """The init_job below refers to epistasis.py"""

    init_job['main_script'] = configuration.main_script#is this used????
    print("11a) GRIDE Executable for epistasis.py is defined!!! and added to job")
    job_cmds = ['python '+ configuration.main_script]
#	"""job_cmds = python epistasis.py"""
    init_job['commands'] = job_cmds
    """ADD NEW PY-SCRIPTS HER OR CALL THEM IN epistasis.py??????????!!!!!!!!!!!!"""	
#Defines data-file by a split of the data-file path into the list e.g. ['RfilesAndscripts', 'M101comp140210W.sav'] and returns the latter ([-1]) as a string
    init_job['data_file'] = configuration.data_file.split('/')[-1]#only data-file, address i eliminated 
#    init_job['data_file2'] = configuration.data_file2.split('/')[-1]#only data-file2, address i eliminated 
#partion:
    init_job['selection_variable'] = configuration.default_selection_variable_index
#genes
    init_job['gene_list'] = configuration.default_gene_list
#traits
    init_job['trait_list'] = configuration.default_trait_list

    init_job['output_dir'] = configuration.output_dir#this is epistasis/
    init_job["started"] = "---"
    init_job["finished"] = "---"
    init_job["gene_first"] = configuration.gene_first_index
    init_job["gene_last"] = configuration.gene_last_index
    init_job["significance_level"] = configuration.default_significance_level
#example for output but may be superflous


#    init_job['results_dir'] = configuration.resultsdir_prefix_name + project_tag +"/"
#THE ABOVE IS OK, BUT PROBLEMS DO
#new_dir = dest_dir+tar_file_path.split("/")[-1].strip(".tar.gz")
#        if not os.path.exists(new_dir):
#            os.mkdir(new_dir)
    return init_job

"""
def mig_call(func,*args):
    #print args
    #funct = "miglib."+func
    #print func, args
    try:
        out = func(*args)
    except:
        print "Unexpected error:", sys.exc_info()[0]
        raise
        

        return out
#print out
    #exit_code = get_exit_code(out)
    
    #print "exit code", exit_code
    #if exit_code != 0:
    #   raise Exception("MiG Error: \n"+str(func)+":"+str(args)+"\n"+"".join(out))
"""

print("3) Initiation of class Gridepistasis done")
# ##### MAIN ###############

    # Arguments are entered in the order: selectionvariableindex jobsize

if __name__ == '__main__':
    local = False
    debug = False
    if '-local' in sys.argv or '-l' in sys.argv:
        local = True
    if '-debug' in sys.argv or '-d' in sys.argv:
        debug = True
    """A new epistasis are started by calling GriEpistasis"""
    """Then start_epistasis is called"""

    print("4) Script 'start_epistasis' is called")
    NEW_EPISTASIS = GridEpistasis(debug_mode=debug)
#    print("NEW_EPISTASIS")
    NEW_EPISTASIS.start_epistasis(local_mode=local)
    NEW_EPISTASIS.monitor_epistasis()
    NEW_EPISTASIS.clean_up_epistasis()	

